' Normal testing:
'c:\wiki_import\input\ /scan /imgpath c:\wiki_import\images /logpath c:\wiki_import\logs /artpath c:\wiki_import\articles /arturl http://wikirs/meridianbeta/index.php?title=%title% /dbserver wikirs /db wikidb /dblogin root /dbpassword mysql /tableprefix wikibeta_ /linktosource /linkpublictext %filename% /linkprivatetext [[file:\\joekessler2\shared_folder\%filename%]]

Imports System
Imports System.Data
Imports System.Net
Imports System.IO
Imports MySql.Data.MySqlClient

Module Main
    ' Conversion parameters.
    Dim FullyQualifiedFilename As String
    Dim ArticleExtension As String
    Dim ArticleCategory As String
    Dim ArticleTitle As String
    Dim AddArticleWrapper As String
    Dim ImageOutputPath As String
    Dim ArticleOutputPath As String
    Dim ArticleOutputUrl As String
    Dim ImageTagTemplate As String
    Dim ScanMode As Boolean
    Dim ScanOnlyOnce As Boolean
    Dim DeleteSourceDocsAfterConversion As Boolean
    Dim DBServer As String
    Dim DB As String
    Dim DBLogin As String
    Dim DBPassword As String
    Dim TablePrefix As String
    Dim LinkToSource As Boolean
    Dim LinkPublicText As String
    Dim LinkPrivateText As String
    Dim ScrapeFromSite As Boolean
    Dim CrawlSite As Boolean
    Dim SiteLogin As String = ""
    Dim SitePassword As String = ""
    Dim WorkPath As String = "C:\tmp"
    Dim AutoLink As Boolean = False
    Dim Sponsor As String = Nothing
    Dim Logger As LogWriter = New LogWriter

    Sub Main(ByVal ArgList As String())
        Logger.LogLine("=======================================================================")
        Logger.LogLine("WikiWordImport Document Converter")
        Logger.LogLine("Version 1.0a")
        Logger.LogLine("=======================================================================")
        Logger.LogLine()

        Dim ShowHelp As Boolean
        If ArgList.Length = 0 Then
            ShowHelp = True
        ElseIf ArgList(0) = "/?" Then
            ShowHelp = True
        Else
            ShowHelp = False
        End If

        If ShowHelp Then
            Logger.LogLine("Imports a single Word document or scans a directory to import multiple")
            Logger.LogLine("documents at once.  Articles are inserted directly into the MediaWiki")
            Logger.LogLine("database.")
            Logger.LogLine()
            Logger.LogLine("Usage:")
            Logger.LogLine("WikiWordImport <parameter file>")
        Else
            Dim imp As Impersonator = Nothing
            Try
                ' Construct our importer instance.
                Dim Arg As String
                Dim NextParm As String

                FullyQualifiedFilename = ArgList(0)
                ArticleExtension = ".txt"
                ArticleCategory = Nothing
                ArticleTitle = Nothing
                AddArticleWrapper = False
                ImageOutputPath = "c:\wiki_import\"
                ImageTagTemplate = "[[image:$base$]]"
                ArticleOutputPath = "c:\wiki_import\"
                ArticleOutputUrl = Nothing
                ScanMode = False
                ScanOnlyOnce = False
                DeleteSourceDocsAfterConversion = False

                DBServer = Nothing
                DB = Nothing
                DBLogin = Nothing
                DBPassword = Nothing
                TablePrefix = Nothing
                SiteLogin = Nothing
                SitePassword = Nothing

                ' Read our parm file and produce a command-line equivalent.
                Dim PracticalArgList As ArrayList

                PracticalArgList = ParseCommandFile("DefaultSettings.ini")
                PracticalArgList.AddRange(ParseCommandFile(ArgList(0)))

                ' Iterate over each parm.
                Logger.LogLine("[Debug]")
                NextParm = Nothing
                For Each Arg In PracticalArgList
                    Logger.LogLine("Arg=" + Arg)

                    ' See if the value is second part of a tagged one.
                    ' The preceding parm has a  '/' prefix).
                    If (NextParm = "xml") Then
                        AddArticleWrapper = ParseBool(Arg)
                    End If

                    If (NextParm = "autolink") Then
                        AutoLink = ParseBool(Arg)
                    End If

                    If (NextParm = "linktosource") Then
                        LinkToSource = ParseBool(Arg)
                    End If

                    If (NextParm = "scan") Then
                        ScanMode = ParseBool(Arg)
                    End If

                    If (NextParm = "scanonce") Then
                        ScanOnlyOnce = ParseBool(Arg)
                        If (ScanOnlyOnce) Then
                            ScanMode = True
                        End If
                    End If

                    If (NextParm = "deleteafter") Then
                        DeleteSourceDocsAfterConversion = ParseBool(Arg)
                    End If

                    If (NextParm = "scrapefrom") Then
                        ScrapeFromSite = ParseBool(Arg)
                    End If

                    If (NextParm = "sponsor") Then
                        Sponsor = Arg
                    End If

                    If (NextParm = "cat") Then
                        ArticleCategory = Arg.Trim().Replace(" ", "_")
                    End If

                    If (NextParm = "logpath") Then
                        Logger.LogOutputPath = Arg
                    End If

                    If (NextParm = "crawlsite") Then
                        CrawlSite = ParseBool(Arg)
                    End If

                    If (NextParm = "workpath") Then
                        WorkPath = Arg
                    End If

                    If (NextParm = "dbserver") Then
                        DBServer = Arg
                    End If

                    If (NextParm = "db") Then
                        DB = Arg
                    End If

                    If (NextParm = "dblogin") Then
                        DBLogin = Arg
                    End If

                    If (NextParm = "dbpassword") Then
                        DBPassword = Arg
                    End If

                    If (NextParm = "tableprefix") Then
                        TablePrefix = Arg
                    End If

                    If (NextParm = "title") Then
                        ArticleTitle = Arg
                    End If

                    If (NextParm = "imgpath") Then
                        ImageOutputPath = Arg
                    End If

                    If (NextParm = "imgtag") Then
                        ImageTagTemplate = Arg
                    End If

                    If (NextParm = "artpath") Then
                        ArticleOutputPath = Arg
                    End If

                    If (NextParm = "arturl") Then
                        ArticleOutputUrl = Arg
                    End If

                    If (NextParm = "articleextension") Then
                        ArticleExtension = Arg
                    End If

                    If (NextParm = "linkpublictext") Then
                        LinkPublicText = Arg
                    End If

                    If (NextParm = "linkprivatetext") Then
                        LinkPrivateText = Arg
                    End If

                    If (NextParm = "sitelogin") Then
                        SiteLogin = Arg
                    End If

                    If (NextParm = "sitepassword") Then
                        SitePassword = Arg
                    End If

                    If (NextParm = "docpath") Then
                        FullyQualifiedFilename = Arg
                    End If

                    NextParm = Nothing

                    ' look for prefixes and store what type for the next iteration.
                    If Left(Arg, 1) = "/" Then
                        NextParm = Right(Arg, Len(Arg) - 1)
                    End If
                Next

                ' Make sure all paths end with '\'.
                If (Right(ImageOutputPath, 1) <> "\") Then
                    ImageOutputPath = ImageOutputPath + "\"
                End If

                If (Right(ArticleOutputPath, 1) <> "\") Then
                    ArticleOutputPath = ArticleOutputPath + "\"
                End If

                If (Right(Logger.LogOutputPath, 1) <> "\") Then
                    Logger.LogOutputPath = Logger.LogOutputPath + "\"
                End If

                ' Decode the site password.
                Try
                    Dim enc As New System.Text.ASCIIEncoding
                    Dim pwBytes As Byte() = Convert.FromBase64String(SitePassword)
                    SitePassword = enc.GetString(pwBytes)
                Catch ' May fail if not Base 64.  Plain text password is okay so we'll let it fly.
                End Try

                ' Impersonate as needed.
                If (SiteLogin <> "") Then
                    Logger.LogLine("Impersonating: " + SiteLogin)
                    imp = New Impersonator
                    If (Not imp.ImpersonateStart("cdw-1", SiteLogin, SitePassword)) Then
                        Throw New Exception("Error impersonating cdw-1\" + SiteLogin + "'")
                    End If
                End If

                ' Convert away!
                ContinualScan()
            Catch ex As Exception
                Logger.LogLine()
                Logger.LogLine("Error!")
                Logger.LogLine(ex.Message)
                Logger.LogLine()
                Logger.LogLine(ex.StackTrace)
            Finally
                If Not imp Is Nothing Then
                    imp.ImpersonateStop()
                End If
            End Try
        End If
    End Sub

    Function ParseArgs(ByVal ArgList As String(), ByVal NeedToRescanParms As Boolean) As ArrayList
        Dim WorkingParmList As ArrayList
        Dim Arg As String

        WorkingParmList = New ArrayList

        ' Go over each parm.
        For Each Arg In ArgList
            ' See if we need to adjust the parm.
            If (Not NeedToRescanParms) Then
                WorkingParmList.Add(Arg)
            Else
                WorkingParmList.AddRange(BreakOutParms(Arg))
            End If
        Next

        'If (UBound(ArgList) = 1) Then
        'End If
        ParseArgs = WorkingParmList
    End Function

    Function BreakOutParms(ByVal Text As String) As ArrayList
        Dim WorkingSectionList As ArrayList
        Dim inQuote As Boolean
        Dim WorkingText As String
        Dim ch As Char

        Text = Text + " " ' Make sure we break at end
        WorkingText = ""
        inQuote = False
        WorkingSectionList = New ArrayList

        ' Iterate through and break based on whitespaces, sans quoted things.
        For Each ch In Text
            ' Toggle for quotes.
            If (ch = Chr(34)) Then
                inQuote = Not inQuote
            Else
                ' In a quote - just copy chars over to the result.
                If (inQuote) Then
                    WorkingText = WorkingText + ch
                    ' Break on a whitespace.
                Else
                    If (Char.IsWhiteSpace(ch)) Then
                        WorkingSectionList.Add(Trim(WorkingText))
                        WorkingText = ""
                    Else
                        WorkingText = WorkingText + ch
                    End If

                End If

            End If
        Next

        ' Return our split text.
        BreakOutParms = WorkingSectionList
    End Function

    Sub ContinualScan()
        Dim ShouldStillContinue As Boolean
        Dim ScannedFilenames As String() = Nothing
        Dim FilenamesAlreadyTried As ArrayList
        Dim connStr As String
        Dim conn As MySqlConnection
        Dim totalImportCount As Integer = 0

        ' Make sure we've got enough data to connect to the DB.
        'Logger.LogLine("Here!  zwa")
        If (DBServer = Nothing) Or _
           (DB = Nothing) Or _
           (DBLogin = Nothing) Or _
           (DBPassword = Nothing) Or _
           (TablePrefix = Nothing) Then
            conn = Nothing
        Else
            ' Log into MySQL.
            Logger.LogLine("Logging into MySQL (db=" + DB + ") as " + DBLogin)
            connStr = String.Format("server={0};user id={1}; password={2}; database={3}; pooling=false", DBServer, DBLogin, DBPassword, DB)

            conn = New MySqlConnection(connStr)
            conn.Open()
            conn.ChangeDatabase(DB)
        End If


        ' Look for files in the directory.
        Logger.LogLine("Scanning for documents in " + FullyQualifiedFilename)
        Logger.LogLine()

        FilenamesAlreadyTried = New ArrayList
        ShouldStillContinue = True
        While (ShouldStillContinue)
            ' Ping the DB to keep the connection fresh.
            PingDB(conn)

            ' Look for docs that need to be converted.
            If Not ScanMode Then
                ' Single doc mode - use the input path as a qualified filename.
                Dim SingleFileArray As String()
                ReDim SingleFileArray(1)

                SingleFileArray(0) = FullyQualifiedFilename
                ScannedFilenames = SingleFileArray
            Else
                ' Scan mode - get a list of files in the source path.
                Dim ShouldScan As Boolean = False
                If (Not ScrapeFromSite) Or (ScannedFilenames Is Nothing) Then
                    ShouldScan = True
                ElseIf (ScannedFilenames.Length = 0) Then
                    ShouldScan = True
                End If

                If (ShouldScan) Then
                    ScannedFilenames = FindDocsToConvert(FullyQualifiedFilename)
                End If
            End If

            ' Convert each file in the scan path.
            Dim DocToConvert As String
            For Each DocToConvert In ScannedFilenames
                Dim LogFilename As String = Nothing
                Dim DocProcessedSuccessfully As Boolean = False
                Dim MasterSourceFilename As String = DocToConvert
                Dim LocalFilename As String = DocToConvert
                Dim ArticleText As String = ""

                ' Ping the DB to keep the connection fresh.
                Try
                    PingDB(conn)
                Catch
                    ' Log into MySQL.
                    Logger.LogLine("Unable to ping DB.  Logging in again...")
                    conn = New MySqlConnection(connStr)
                    conn.Open()
                    conn.ChangeDatabase(DB)
                End Try

                ' Don't process the same document more than once.
                If (DocToConvert.EndsWith(".vsd")) Then
                    DocToConvert = (DocToConvert + " ").Trim
                End If

                ' Derive a unique identifier for this file and its timestamp.  Used
                ' to determine if it has already been imported.
                Dim FileIdentifier As String
                Try
                    FileIdentifier = DocToConvert + ":" + File.GetLastWriteTime(DocToConvert)
                Catch ex As Exception
                    FileIdentifier = DocToConvert
                End Try

                Try
                    ' If this version of the file has already been converted/
                    If (Not FilenamesAlreadyTried.Contains(FileIdentifier)) Then
                        totalImportCount = totalImportCount + 1

                        If (ScrapeFromSite) Then
                            ' Hack out the relative file name.
                            Dim relPathSep As Int32

                            relPathSep = InStrRev(DocToConvert, "\")
                            If (relPathSep <= 0) Then
                                relPathSep = InStrRev(DocToConvert, "/")
                            End If

                            Dim relFilename As String
                            If (relPathSep > 0) Then
                                relFilename = Right(DocToConvert, DocToConvert.Length - relPathSep)
                            Else
                                relFilename = DocToConvert
                            End If
                            relFilename = relFilename.Replace("%20", " ")

                            ' Retrieves the file from the specified URL
                            Dim objWC As New System.Net.WebClient
                            Dim cred As ICredentials

                            ' Issue the web request.
                            If (Not SiteLogin Is Nothing) Then
                                objWC.Credentials = New Net.NetworkCredential(SiteLogin, SitePassword)
                            End If

                            ' Form the local filename.
                            Dim ext As String
                            Dim perPos As Integer = InStrRev(relFilename, ".")

                            If (perPos > 0) Then
                                ext = Right(relFilename, relFilename.Length - perPos)
                            Else
                                ext = ""
                            End If

                            If WorkPath.EndsWith("\") Then
                                LocalFilename = WorkPath + "F" + Guid.NewGuid().ToString() + "." + ext
                            Else
                                LocalFilename = WorkPath + "\F" + Guid.NewGuid().ToString() + "." + ext
                            End If

                            Logger.LogLine("Local filename is: " + LocalFilename)

                            ' Copy the file!
                            File.Delete(LocalFilename)
                            objWC.DownloadFile(MasterSourceFilename, LocalFilename)
                        End If

                        ' Start a new log for this file.
                        Logger.LogLine("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -")
                        Logger.LogLine("Importing file #" + totalImportCount.ToString())
                        Logger.DumpLog()

                        Dim ActualCategory As String
                        Dim ActualTitle As String
                        Dim shouldAutoLink As Boolean
                        Dim FinalImageFileList As ArrayList

                        ' Strip off the category name if one is given in the filename.
                        ActualCategory = ArticleCategory
                        ActualTitle = ArticleTitle
                        ArticleText = ""
                        FinalImageFileList = New ArrayList

                        Dim attribScanDocName As String = DocToConvert
                        Dim lastPathSep As Integer
                        Dim beginBracketPos As Integer
                        Dim endBracketPos As Integer
                        Dim periodPos As Integer
                        Dim leftPart As String
                        Dim rightPart As String
                        Dim relativeFilename As String

                        ' See if we should autolink the document.
                        Dim authorLinkStr As String = GetMetadataValue(attribScanDocName, "author")
                        Dim titleLinkStr As String = GetMetadataValue(attribScanDocName, "title")
                        Dim categoryLinkStr As String = GetMetadataValue(attribScanDocName, "category")
                        Dim departmentLinkStr As String = GetMetadataValue(attribScanDocName, "department")
                        'Dim abstractLinkStr As String = GetMetadataValue(attribScanDocName, "abstract")
                        Dim autoLinkStr As String = GetMetadataValue(attribScanDocName, "autolink")

                        If (autoLinkStr = "true") Or (AutoLink = True) Then
                            shouldAutoLink = True
                        Else
                            shouldAutoLink = False
                        End If

                        ' Look for a category specifier.
                        If Not (categoryLinkStr = Nothing) Then
                            ActualCategory = categoryLinkStr
                        End If

                        ' Get the relative filename.
                        lastPathSep = InStrRev(attribScanDocName, "/")
                        If (lastPathSep > 0) Then
                            relativeFilename = Right(attribScanDocName, attribScanDocName.Length - lastPathSep)
                        Else
                            relativeFilename = attribScanDocName
                        End If

                        lastPathSep = InStrRev(relativeFilename, "\")
                        If (lastPathSep > 0) Then
                            relativeFilename = Right(relativeFilename, relativeFilename.Length - lastPathSep)
                        Else
                            relativeFilename = relativeFilename
                        End If

                        ' Skip any file starting with a "~".
                        If relativeFilename.StartsWith("~") Then
                            GoTo NextFile
                        End If

                        ' Remove the extension from the article title.
                        periodPos = InStrRev(attribScanDocName, ".")
                        If (periodPos > 0) Then
                            attribScanDocName = Left(attribScanDocName, periodPos - 1)
                        End If

                        ' Remove the path name from the article title.
                        lastPathSep = InStrRev(attribScanDocName, "\")
                        ActualTitle = Right(attribScanDocName, Len(attribScanDocName) - lastPathSep)
                        lastPathSep = InStrRev(ActualTitle, "/")
                        If (lastPathSep > 0) Then
                            ActualTitle = Right(ActualTitle, Len(ActualTitle) - lastPathSep)
                        End If

                        ' Remove the path name from the local filename.
                        Dim LocalBaseFilename As String
                        lastPathSep = InStrRev(LocalFilename, "\")
                        LocalBaseFilename = Right(LocalFilename, Len(LocalFilename) - lastPathSep)

                        ' Remove metadata from the filename.
                        ActualTitle = BlankOutDelimitedText(ActualTitle, "[", "]").Trim()
                        ActualTitle = ActualTitle.Replace("%20", " ")

                        System.Console.WriteLine("Article base title will be: " + ActualTitle)

                        ' Determine the file type.
                        Dim FileExt As String
                        FileExt = Right(DocToConvert, 3).ToLower()

                        ' --- Common handling ---------------------------------------
                        If LinkToSource Then
                            Dim PublicLink As String
                            Dim PrivateLink As String

                            If (LinkPublicText Is Nothing) Or (LinkPublicText = "") Then
                                PublicLink = relativeFilename
                            Else
                                PublicLink = LinkPublicText.Replace("{filename}", relativeFilename)
                                PublicLink = PublicLink.Replace("{filename_esc}", relativeFilename.Replace(" ", "%20"))
                                PublicLink = PublicLink.Replace("{localfilename}", LocalBaseFilename.Replace("%20", " "))
                                PublicLink = PublicLink.Replace("{localfilename_esc}", LocalBaseFilename.Replace(" ", "%20"))
                                PublicLink = PublicLink.Replace("{href}", MasterSourceFilename)
                            End If

                            If (LinkPrivateText Is Nothing) Or (LinkPrivateText = "") Then
                                PrivateLink = MasterSourceFilename.Replace(" ", "%20")
                            Else
                                PrivateLink = LinkPrivateText.Replace("{filename}", relativeFilename)
                                PrivateLink = PrivateLink.Replace("{localfilename}", LocalBaseFilename.Replace("%20", " "))
                                PrivateLink = PrivateLink.Replace("{localfilename_esc}", LocalBaseFilename.Replace(" ", "%20"))
                                PrivateLink = PrivateLink.Replace("{filename_esc}", relativeFilename.Replace(" ", "%20"))
                                PrivateLink = PrivateLink.Replace("{href}", MasterSourceFilename.Replace(" ", "%20"))
                            End If

                            ArticleText = "<table border=0 width=100%><tr bgcolor=#ccccff><td>"
                            ArticleText = ArticleText + "The content for this article comes from the source listed here:<br><br>"

                            ArticleText = ArticleText + "<table border=0 cellspacing=0>"
                            ArticleText = ArticleText + "<tr bgcolor=#ccccff>"
                            ArticleText = ArticleText + "<td>"
                            ArticleText = ArticleText + "&nbsp;&nbsp;&nbsp;&nbsp;<b>Source:</b> "
                            ArticleText = ArticleText + "</td>"
                            ArticleText = ArticleText + "<td>"
                            ArticleText = ArticleText + "&nbsp;&nbsp;" + PrivateLink
                            If (ScrapeFromSite) Then
                                ArticleText = ArticleText + " from [" + FullyQualifiedFilename + " this site]<br>"
                            Else
                                ArticleText = ArticleText + " from [file:" + FullyQualifiedFilename + " this location]<br>"
                            End If
                            ArticleText = ArticleText + "</td>"
                            ArticleText = ArticleText + "</tr>"

                            'If (FileExt = "doc") Or (FileExt = "rtf") Or (FileExt = "txt") Then
                            ArticleText = ArticleText + "<tr bgcolor=#ccccff>"
                            ArticleText = ArticleText + "<td>"
                            ArticleText = ArticleText + "&nbsp;&nbsp;&nbsp;&nbsp;<b>Sync Time:</b> "
                            ArticleText = ArticleText + "</td>"
                            ArticleText = ArticleText + "<td>"
                            ArticleText = ArticleText + "&nbsp;&nbsp;" + Date.Now.ToString()
                            ArticleText = ArticleText + "</td>"
                            ArticleText = ArticleText + "</tr>"
                            'End If

                            If (Not Sponsor Is Nothing) Then
                                ArticleText = ArticleText + "<tr bgcolor=#ccccff>"
                                ArticleText = ArticleText + "<td>"
                                ArticleText = ArticleText + "&nbsp;&nbsp;&nbsp;&nbsp;<b>Sponsor:</b> "
                                ArticleText = ArticleText + "</td>"
                                ArticleText = ArticleText + "<td>"
                                ArticleText = ArticleText + "&nbsp;&nbsp;" + Sponsor + " <i>(User who linked this site to the Wiki)</i>"
                                ArticleText = ArticleText + "</td>"
                                ArticleText = ArticleText + "</tr>"
                            End If
                            ArticleText = ArticleText + "</table>"

                            ArticleText = ArticleText + "<br>"
                            ArticleText = ArticleText + "If you edit this article in the Wiki your changes may be lost.  "
                            ArticleText = ArticleText + "To make changes, go to the source site instead by following the link provided above.<br>"
                            ArticleText = ArticleText + "</td></tr></table>"
                            ArticleText = ArticleText + "<!-- LinkSource=<" + PrivateLink + "> -->"
                            ArticleText = ArticleText + "<!-- LinkSourceSite=<" + LinkPrivateText + "> -->"
                            'ArticleText = ArticleText + "<!-- LinkSourceTimestamp=<" + File.GetLastWriteTimeUtc(DocToConvert).Ticks.ToString() + "> -->"
                        End If

                        ' --- Handle a Word file ------------------------------------
                        Dim ImportThreadTimedOut As Boolean = False

                        If (FileExt = "doc") Or _
                           (FileExt = "rtf") Or _
                           (FileExt = "ppt") Or _
                           (FileExt = "vsd") Or _
                           (FileExt = "txt") Then

                            ' Construct our importer instance.
                            Dim Importer As WordImporter

                            Importer = New WordImporter
                            Importer.FullyQualifiedFilename = DocToConvert
                            Importer.ArticleExtension = ArticleExtension
                            Importer.ArticleCategory = ActualCategory
                            Importer.ArticleDepartment = departmentLinkStr
                            Importer.ArticleAuthor = authorLinkStr
                            Importer.ArticleAbstract = ""
                            Importer.shouldAutoLink = shouldAutoLink
                            Importer.ArticleTitle = ActualTitle
                            Importer.AddArticleWrapper = AddArticleWrapper
                            Importer.ImageOutputPath = ImageOutputPath
                            Importer.ImageTagTemplate = ImageTagTemplate
                            Importer.ArticleOutputPath = ArticleOutputPath
                            Importer.Logger = Logger

                            ' Convert the doc!
                            Logger.LogLine("[ " + DateTime.Now.ToString() + " ]")

                            Try
                                ' Open the document!
                                Dim LoadThread As New System.Threading.Thread(AddressOf Importer.LoadDocument)

                                LoadThread.IsBackground = True
                                LoadThread.Start()
                                LoadThread.Join(600000) ' Die in 10 minutes if not returned.
                                If LoadThread.IsAlive Then
                                    Throw New Exception("Timeout while loading document")
                                End If

                                ' Import!
                                Dim ImportThread As New System.Threading.Thread(AddressOf Importer.Import)
                                ImportThread.IsBackground = True
                                ImportThread.Start()

                                Dim TimeoutMS As Integer
                                TimeoutMS = Importer.PageCount * 10000 ' 10 seconds for each page
                                If (TimeoutMS > 600000) Then ' Max of 10 minutes
                                    TimeoutMS = 600000
                                End If

                                Logger.LogLine("Converting " + Importer.FullyQualifiedFilename)
                                Logger.Log(" (timeout in " + (TimeoutMS / 1000).ToString() + " seconds)...")
                                Logger.LogLine()

                                If (Not ImportThread.Join(TimeoutMS)) Then ' Ten minute timeout
                                    Logger.LogLine("Import thread timed out")
                                    ImportThreadTimedOut = True
                                    ImportThread.Abort()
                                End If
                            Finally
                                LogFilename = Importer.ArticleTitle + ".html"
                                ActualTitle = Importer.ArticleTitle
                            End Try

                            Logger.LogLine()
                            Logger.LogLine("Done!")
                            ArticleTitle = Importer.ArticleTitle
                            ArticleText = ArticleText + Importer.ArticleText
                            FinalImageFileList = Importer.FinalImageFileList

                            Importer = Nothing
                        Else
                            ArticleTitle = ActualTitle
                        End If
                    End If

                Catch ex As Exception
                    Logger.LogLine()
                    Logger.LogLine("Error while importing document!")
                    Logger.LogLine(ex.Message)
                    Logger.LogLine()
                    Logger.LogLine(ex.StackTrace)
                Finally
                    ' Record that we've already tried converting this file.
                    FilenamesAlreadyTried.Add(FileIdentifier)
                End Try

                ' Invoke the "article" callback.
                Dim DocWasInsertedIntoDB As Boolean = False
                If (Not conn Is Nothing) And (ArticleText <> "") Then
                    Exec(conn, "START TRANSACTION")

                    Try
                        If Not conn Is Nothing Then
                            DocWasInsertedIntoDB = StoreArticleInDB(conn, TablePrefix, ArticleTitle, ArticleText)
                        End If

                        ' Invoke the "image" callback for each generated image.
                        Dim ImageFilename As String
                        'For Each ImageFilename In FinalImageFileList
                        'StoreImageInDB(conn, TablePrefix, Importer, ImageFilename)
                        'Next

                        ' Commit our changes.
                        Exec(conn, "COMMIT")
                        DocProcessedSuccessfully = DocWasInsertedIntoDB
                    Catch ex As Exception
                        Try
                            Exec(conn, "ROLLBACK")
                        Catch
                        End Try
                        Logger.LogLine(ex.ToString())

                        ' Let it keep going even if there is an error.
                        'Throw ex
                    End Try

                    ' Write link to new article.
                    Logger.LogLine()
                    If DocProcessedSuccessfully Then
                        Logger.LogLine("<font color=green><b>Article was imported successfully!</b></font>")

                        If (Not ArticleOutputUrl = Nothing) Then
                            Dim FinalUrl As String

                            'Logger.Log("Follow this ")
                            'Logger.Log("<a href=" + Replace(ArticleOutputUrl, "%title%", Replace(ActualTitle, " ", "_")) + ">")
                            'Logger.LogLine("link</a> to the imported article")
                        End If
                    Else
                        Logger.LogLine("<font color=red><b>Article could not be imported</b></font>")
                    End If
                End If

                ' Remove the source doc to prevent it from processing again.
                'If DocProcessedSuccessfully And DeleteSourceDocsAfterConversion Then
                If DeleteSourceDocsAfterConversion And (DocToConvert.IndexOf("\\") < 0) Then ' Never delete from shares
                    Try
                        System.IO.File.Delete(DocToConvert)
                        FilenamesAlreadyTried.Remove(DocToConvert) ' Can import again if successful delete.

                    Catch ex As Exception
                        Logger.LogLine("Error deleting source doc:")
                        Logger.LogLine(ex.Message)
                    End Try
                End If

                'System.Console.WriteLine(LogFilename)
                Logger.Write(Replace(LogFilename, " ", "_"))
            Next

            ' If we're only scanning the directory one time, flag that we're 
            ' done after this pass.
            If ScanOnlyOnce Or (Not ScanMode) Then
                ShouldStillContinue = False
            Else
                ' Wait for a short while before converting more.
                System.Threading.Thread.Sleep(1000)
            End If

NextFile:
        End While

        ' Update the "good article" count.
        Logger.LogLine("Correcting 'Good article count'")

        Dim reader As MySqlDataReader
        reader = Query(conn, _
                       "SELECT COUNT(*) FROM " + GetTableName(TablePrefix, "page") + " " _
                     + "LEFT JOIN " + GetTableName(TablePrefix, "revision") + " ON (rev_id=page_latest) " _
                     + "LEFT JOIN " + GetTableName(TablePrefix, "text") + " ON (old_id=rev_text_id) " _
                     + "WHERE page_namespace = 0")


        If (reader.Read()) Then
            Dim count As Integer
            count = reader.GetInt32(0)
            reader.Close()

            Logger.LogLine("There are " + count.ToString() + " 'good articles'")
            Exec(conn, "UPDATE " + GetTableName(TablePrefix, "site_stats") _
                     + " SET ss_good_articles = " + count.ToString() + "," _
                     + "     ss_total_pages = " + count.ToString() _
                     + " WHERE `ss_row_id` = '1' LIMIT 1")
        Else
            reader.Close()
        End If

        ' Log out of MySQL.
        If (Not conn Is Nothing) Then
            Logger.LogLine("Logging out of MySQL")
            conn.Close()
        End If
    End Sub

    Function FindDocsToConvert(ByVal ScanPath As String) As String()
        If (ScrapeFromSite) Then
            Logger.LogLine("Crawling Sharepoint Site...")

            FindDocsToConvert = ScrapeWebSite(ScanPath)
        Else
            ' Include subdirectories.
            If (CrawlSite) Then
                Dim AccumFileList As ArrayList = New ArrayList

                RecursiveFileList(ScanPath, AccumFileList)
                Dim TypedResult As String() = CType(AccumFileList.ToArray(GetType(String)), String())
                Return TypedResult
            Else
                Return System.IO.Directory.GetFiles(ScanPath)
            End If
        End If
    End Function

    Sub RecursiveFileList(ByVal Path As String, ByVal AccumFileList As ArrayList)
        Dim fileList As String() = System.IO.Directory.GetFiles(Path)
        Dim filename As String

        ' Add each file to the accum list.
        For Each Filename In fileList
            AccumFileList.Add(filename)
        Next

        ' Recurse down to subdirs.
        Dim SubDirList As String() = System.IO.Directory.GetDirectories(Path)
        Dim Dirname As String

        For Each Dirname In SubDirList
            RecursiveFileList(Dirname, AccumFileList)
        Next
    End Sub

    Function ScrapeWebSite(ByVal URL As String) As String()
        Dim resultSet As ArrayList = New ArrayList
        Dim sitesVisited As ArrayList = New ArrayList

        InternalScrapeWebSite(URL, resultSet, sitesVisited)
        Dim TypedResult As String() = CType(resultSet.ToArray(GetType(String)), String())
        ScrapeWebSite = TypedResult
    End Function

    Sub InternalScrapeWebSite(ByVal URL As String, ByVal LinkedDocs As ArrayList, ByVal SitesVisited As ArrayList)
        ' Don't visit the same site twice.
        If (SitesVisited.Contains(URL)) Then
            Return
        Else
            SitesVisited.Add(URL)
        End If

        ' Retrieves the HTML from the specified URL
        Dim objWC As New System.Net.WebClient
        Dim cred As ICredentials
        Dim utf8 As System.Text.UTF8Encoding
        Dim html As String
        Dim htmlLower As String

        ' Un-escape chars
        URL = URL.Replace("%20", " ")

        ' Issue the web request.
        If (Not SiteLogin Is Nothing) Then
            objWC.Credentials = New Net.NetworkCredential(SiteLogin, SitePassword)
        End If

        utf8 = New System.Text.UTF8Encoding
        html = utf8.GetString(objWC.DownloadData(URL))

        ' Hack out everything but the file list.
        Dim startPos As Integer = html.IndexOf("Checked Out To")
        If (startPos < 0) Then
            startPos = html.IndexOf("New Folder")
        End If
        If (startPos < 0) Then
            startPos = html.IndexOf("Upload Document")
        End If
        If (startPos < 0) Then
            startPos = html.IndexOf("New Document")
        End If
        If (startPos < 0) Then
            startPos = html.IndexOf("Modified By")
        End If
        If (startPos < 0) Then
            startPos = html.IndexOf("Type Name")
        End If

        html = html.Substring(startPos)

        ' Iterate over every <A>nchor tag.
        htmlLower = html.ToLower()
        Dim anchorPos As Int32
        Dim stillGoing As Boolean
        Dim anyDocsFound As Boolean = False

        anchorPos = 2
        stillGoing = True
        While stillGoing
            Dim hrefPos As Int32
            Dim hrefValue As String

            stillGoing = False
            anchorPos = htmlLower.IndexOf("<a", anchorPos + 1)
            If anchorPos > 0 Then
                ' Find the HREF value
                hrefValue = Nothing

                hrefPos = htmlLower.IndexOf("href=" + Chr(34), anchorPos)
                If (hrefPos > 0) Then
                    Dim endHRefPos As Int32

                    endHRefPos = htmlLower.IndexOf(Chr(34), hrefPos + 6)
                    If (endHRefPos >= 0) Then
                        hrefValue = html.Substring(hrefPos + 6, endHRefPos - hrefPos - 6)
                        stillGoing = True

                    End If
                End If

                ' A link was found.
                If Not hrefValue Is Nothing Then
                    Dim hrefCompare As String
                    hrefCompare = hrefValue.ToLower
                    Dim addedAsDoc As Boolean = False

                    If (hrefValue.StartsWith("http:")) Then

                        If hrefCompare.EndsWith(".doc") Or _
                           hrefCompare.EndsWith(".rtf") Or _
                           hrefCompare.EndsWith(".txt") Or _
                           hrefCompare.EndsWith(".xls") Or _
                           hrefCompare.EndsWith(".vsd") Or _
                           hrefCompare.EndsWith(".pdf") Or _
                           hrefCompare.EndsWith(".ppt") Then
                            If (Not LinkedDocs.Contains(hrefValue)) Then
                                LinkedDocs.Add(hrefValue)
                                addedAsDoc = True
                                anyDocsFound = True
                            End If
                        End If
                    End If
                End If
            End If
        End While

        ' Crawl down sub-sites.
        If CrawlSite Then
            stillGoing = True
            anchorPos = 0
            While stillGoing
                Dim hrefValue As String
                Dim scanFor As String = "javascript:SubmitFormPost(" + Chr(34)
                stillGoing = False

                anchorPos = html.IndexOf(scanFor, anchorPos + 1)
                If anchorPos > 0 Then
                    ' Find the HREF value
                    hrefValue = Nothing

                    Dim endHRefPos As Integer = html.IndexOf(Chr(34), anchorPos + scanFor.Length + 1)
                    If (endHRefPos >= 0) Then
                        hrefValue = html.Substring(anchorPos + scanFor.Length, endHRefPos - anchorPos - scanFor.Length)

                        If (hrefValue.StartsWith("http:")) And (hrefValue.IndexOf("RootFolder=") >= 0) Then
                            InternalScrapeWebSite(hrefValue, LinkedDocs, SitesVisited)
                        End If
                        stillGoing = True
                    End If
                End If
            End While
        End If
    End Sub

    Function StoreArticleInDB(ByVal conn As MySqlConnection, ByVal TablePrefix As String, ByVal ArticleTitle As String, ByVal ArticleText As String) As Boolean
        Dim reader As MySqlDataReader
        Dim HashedTitle As String
        Dim PageID As Integer
        Dim ArticleExists As Boolean
        Dim ArticleText2 As String

        Logger.LogLine("Storing article in DB")

        ' Get the title with special characters hashed as needed.
        HashedTitle = ArticleTitle
        HashedTitle = Replace(HashedTitle, " ", "_")

        ' Normalize article text.
        ArticleText = Replace(ArticleText, Chr(13), Chr(10))
        'ArticleText = Trim(ArticleText)5

        ' Add the footer at the bottom.
        ArticleText = ArticleText + Chr(10) + Chr(10) + "{{footer}}"

        ' See if the article already exists!
        reader = Query(conn, "select page_id from " + GetTableName(TablePrefix, "page") + _
                             " where page_title='" + HashedTitle + "'")

        If (reader.Read()) Then
            PageID = reader.GetInt32(0)
            reader.Close()

            ' Get the existing article text.
            Dim SQL As String
            Dim ExistingText As String

            SQL = "select old_text from " + GetTableName(TablePrefix, "text") + ", "
            SQL = SQL + GetTableName(TablePrefix, "revision") + ", "
            SQL = SQL + GetTableName(TablePrefix, "page")
            SQL = SQL + " where page_id=" + PageID.ToString()
            SQL = SQL + "   and rev_id=page_latest and rev_text_id=old_id"

            reader = Query(conn, SQL)
            If (reader.Read()) Then
                ExistingText = reader.GetString(0)
            Else
                ExistingText = ""
            End If
            reader.Close()

            ' An auto-generated article should never wipe out a regular one.
            If (ScrapeFromSite) Then ' Auto-generated
                Dim linkToPos As Integer
                linkToPos = ExistingText.IndexOf("<!-- LinkSource=<")
                If (linkToPos >= 0) Then ' This already was auto-generated.
                    StoreArticleInDB = True
                Else
                    StoreArticleInDB = False
                End If
            Else
                StoreArticleInDB = True
            End If

            If StoreArticleInDB Then
                Logger.LogLine("Article already exists - deleting existing version")
                DeleteExistingArticle(conn, PageID)
                StoreArticleInDB = True
            Else
                Logger.LogLine("Article already exists - not overwriting!")
            End If
        End If

        ' Insert the new article.
        Logger.LogLine("Storing new article")

        reader.Close()
        PageID = InsertNewArticle(conn, TablePrefix, ArticleText, ArticleTitle, HashedTitle)
        StoreArticleInDB = True
    End Function

    Sub StoreImageInDB(ByVal conn As MySqlConnection, ByVal TablePrefix As String, ByVal Importer As WordImporter, ByVal ImageFilename As String)
    End Sub

    Function GetTableName(ByVal TablePrefix As String, ByVal TableName As String) As String
        GetTableName = TablePrefix + TableName
    End Function

    Function Query(ByVal conn As MySqlConnection, ByVal QueryStr As String) As MySqlDataReader
        Dim cmd As MySqlCommand

        cmd = conn.CreateCommand()
        cmd.CommandText = QueryStr
        Query = cmd.ExecuteReader()
    End Function

    Function Exec(ByVal conn As MySqlConnection, ByVal SQL As String) As Integer
        Dim cmd As MySqlCommand

        cmd = conn.CreateCommand()
        cmd.CommandText = SQL
        Exec = cmd.ExecuteNonQuery()
    End Function

    Function InsertNewArticle(ByVal conn As MySqlConnection, ByVal TablePrefix As String, ByVal ArticleText As String, ByVal ArticleTitle As String, ByVal HashedTitle As String) As Integer
        Dim reader As MySqlDataReader
        Dim SQL As String
        Dim PageID As Integer
        Dim RevisionID As Integer
        Dim TextID As Integer
        Dim TotalPages As Integer
        Dim TotalGoodPages As Integer

        Logger.LogLine("Storing article in DB")

        ' Insert the text of the page.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "text") _
             + "(old_text," _
             + "old_flags)" _
             + " values(?text, ?flags)")
        'Exec(conn, SQL)

        Dim cmd As MySqlCommand
        Dim parmPageID As MySqlParameter
        Dim parmArticleText As MySqlParameter
        Dim parmFlags As MySqlParameter

        cmd = conn.CreateCommand()
        cmd.CommandText = SQL

        parmArticleText = cmd.CreateParameter()
        parmFlags = cmd.CreateParameter()

        parmArticleText.ParameterName = "text"
        parmArticleText.Value() = ArticleText
        parmArticleText.MySqlDbType() = MySqlDbType.String

        parmFlags.ParameterName = "flags"
        parmFlags.Value() = "utf-8"
        parmFlags.MySqlDbType() = MySqlDbType.String

        cmd.Parameters().Add(parmArticleText)
        cmd.Parameters().Add(parmFlags)
        cmd.ExecuteNonQuery()

        ' Get the newly generated text ID.
        reader = Query(conn, "select max(old_id) from " + GetTableName(TablePrefix, "text"))
        reader.Read()
        TextID = reader.GetInt32(0)
        reader.Close()

        ' Insert a page record.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "page") _
            + "(page_id," _
            + "page_namespace," _
            + "page_title," _
            + "page_restrictions," _
            + "page_counter," _
            + "page_is_redirect," _
            + "page_is_new," _
            + "page_random," _
            + "page_touched," _
            + "page_latest," _
            + "page_len)" _
            + " values({0}, {1}, '{2}', '{3}', {4}, {5}, {6}, {7}, {8}, {9}, {10})", _
            PageID, 0, HashedTitle, "", 0, "false", "false", Rnd(), GetCurrentDate(), 0, Len(ArticleText))

        Exec(conn, SQL)

        ' Get the newly generated page ID.
        reader = Query(conn, "select max(page_id) from " + GetTableName(TablePrefix, "page"))
        reader.Read()
        PageID = reader.GetInt32(0)
        reader.Close()

        ' Insert the search index record.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "searchindex") _
            + "(si_page," _
            + "si_title," _
            + "si_text)" _
            + " values({0}, '{1}', '{2}')", _
            PageID, ArticleTitle, RemoveNonAlphaNumeric(ArticleText))

        Exec(conn, SQL)

        ' Insert the revision record.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "revision") _
            + "(rev_page," _
            + "rev_text_id," _
            + "rev_comment," _
            + "rev_user," _
            + "rev_user_text," _
            + "rev_timestamp," _
            + "rev_minor_edit," _
            + "rev_deleted)" _
            + " values({0}, {1}, '{2}', '{3}', '{4}', '{5}', {6}, {7})", _
            PageID, TextID, "", "0", "MediaWiki default", GetCurrentDate(), False, False)

        Exec(conn, SQL)

        ' Get the newly generated revision id.
        reader = Query(conn, "select max(rev_id) from " + GetTableName(TablePrefix, "revision"))
        reader.Read()
        RevisionID = reader.GetInt32(0)
        reader.Close()

        ' Update the page_latest in the page table since revision_id was not know at insert time.
        SQL = String.Format("update " + GetTableName(TablePrefix, "page") _
            + " set page_latest={0} " _
            + " where page_id={1}", _
            RevisionID, PageID)

        Exec(conn, SQL)

        ' Insert the category record.
        If (Not ArticleCategory = Nothing) Then
            SQL = String.Format("insert into " + GetTableName(TablePrefix, "categorylinks") _
                + "(cl_from," _
                + "cl_to," _
                + "cl_sortkey," _
                + "cl_timestamp)" _
                + " values({0}, '{1}', '{2}', NOW())", _
                PageID, ArticleCategory, ArticleTitle) ', "Now.ToString("yyyy-mm-dd hh:mm:ss"))

            Exec(conn, SQL)
        End If


        ' Tie to autolink category if appropriate.
        If (InStr(ArticleText, "[[autolink]]") > 0) Then
            SQL = String.Format("insert into " + GetTableName(TablePrefix, "pagelinks") _
                + "(pl_from,pl_namespace,pl_title)" _
                + " values({0},0,'Autolink')", _
                PageID)

            Exec(conn, SQL)
        End If
    End Function

    Sub PingDB(ByVal conn As MySqlConnection)
        Dim SQL As String
        SQL = "select 1 from " + GetTableName(TablePrefix, "categorylinks")
        Exec(conn, SQL)
    End Sub

    Function GetCurrentDate() As String
        GetCurrentDate = Now.ToString("yyyyMMddhhmmss")
    End Function

    Function GetEscapedArticleText(ByVal ArticleText As String) As String
        Dim buf As String
        Dim i As Integer
        Dim ch As Char

        ' Look for special chars in the input.
        For i = 0 To (Len(ArticleText) - 1)
            ch = ArticleText.Chars(i)

            ' Double up certain chars.
            If (ch = Chr(39)) Then
                buf = buf + ch
            End If
            buf = buf + ch
        Next

        ' Set our normalized result.
        GetEscapedArticleText = Trim(buf)
    End Function

    Function RemoveNonAlphaNumeric(ByVal ArticleText As String) As String
        Dim buf As String
        Dim i As Integer
        Dim ch As Char

        ' Look for special chars in the input.
        For i = 0 To (Len(ArticleText) - 1)
            ch = ArticleText.Chars(i)

            ' Double up certain chars.
            If Char.IsLetterOrDigit(ch) Then
                buf = buf + ch
            Else
                buf = buf + " "
            End If
        Next

        ' Set our normalized result.
        RemoveNonAlphaNumeric = Trim(buf)
    End Function

    Sub UpdateExistingArticle(ByVal conn As MySqlConnection, ByVal TablePrefix As String, ByVal ArticleText As String, ByVal ArticleTitle As String, ByVal PageID As Integer, ByVal HashedTitle As String)
        Dim reader As MySqlDataReader
        Dim SQL As String
        Dim RevisionID As Integer
        Dim TextID As Integer

        Logger.LogLine("Storing article in DB")

        ' Insert the text of the page.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "text") _
             + "(old_text," _
             + "old_flags)" _
             + " values(?text, ?flags)")
        'Exec(conn, SQL)

        Dim cmd As MySqlCommand
        Dim parmPageID As MySqlParameter
        Dim parmArticleText As MySqlParameter
        Dim parmFlags As MySqlParameter

        cmd = conn.CreateCommand()
        cmd.CommandText = SQL

        parmArticleText = cmd.CreateParameter()
        parmFlags = cmd.CreateParameter()

        parmArticleText.ParameterName = "text"
        parmArticleText.Value() = ArticleText
        parmArticleText.MySqlDbType() = MySqlDbType.String

        parmFlags.ParameterName = "flags"
        parmFlags.Value() = "utf-8"
        parmFlags.MySqlDbType() = MySqlDbType.String

        cmd.Parameters().Add(parmArticleText)
        cmd.Parameters().Add(parmFlags)
        cmd.ExecuteNonQuery()

        ' Get the newly generated text ID.
        reader = Query(conn, "select max(old_id) from " + GetTableName(TablePrefix, "text"))
        reader.Read()
        TextID = reader.GetInt32(0)
        reader.Close()

        ' Update the existing page record.
        SQL = String.Format("update " + GetTableName(TablePrefix, "page") + " set " _
            + "page_title = '{0}'," _
            + "page_touched = {1}," _
            + "page_latest = {2}," _
            + "page_len = {3}" _
            + " where page_id = {4}", _
            HashedTitle, GetCurrentDate(), TextID, Len(ArticleText), PageID) ' TODO: TextID should be RevisionID?

        Exec(conn, SQL)

        ' Update the search index record.
        SQL = String.Format("update " + GetTableName(TablePrefix, "searchindex") + " set " _
            + "si_title = '{0}'," _
            + "si_text = '{1}' " _
            + " where si_page = {2}", _
            ArticleTitle, RemoveNonAlphaNumeric(ArticleText), PageID)

        Exec(conn, SQL)

        ' Insert the revision record.
        SQL = String.Format("insert into " + GetTableName(TablePrefix, "revision") _
            + "(rev_page," _
            + "rev_text_id," _
            + "rev_comment," _
            + "rev_user," _
            + "rev_user_text," _
            + "rev_timestamp," _
            + "rev_minor_edit," _
            + "rev_deleted)" _
            + " values({0}, {1}, '{2}', '{3}', '{4}', '{5}', {6}, {7})", _
            PageID, TextID, "", "0", "MediaWiki default", GetCurrentDate(), False, False)

        Exec(conn, SQL)

        ' Get the newly generated revision id.
        reader = Query(conn, "select max(rev_id) from " + GetTableName(TablePrefix, "revision"))
        reader.Read()
        RevisionID = reader.GetInt32(0)
        reader.Close()

        ' Update the page_latest in the page table since revision_id was not known at insert time.
        SQL = String.Format("update " + GetTableName(TablePrefix, "page") _
            + " set page_latest={0} " _
            + " where page_id={1}", _
            RevisionID, PageID)

        Exec(conn, SQL)

        ' Insert the category record.
        If (Not ArticleCategory = Nothing) Then
            ' Remove any existing category records for this sucker.
            SQL = String.Format("delete from " + GetTableName(TablePrefix, "categorylinks") _
                + " where cl_from = {0}", _
                PageID)

            Exec(conn, SQL)

            ' Insert the new category record.
            SQL = String.Format("insert into " + GetTableName(TablePrefix, "categorylinks") _
                + "(cl_from," _
                + "cl_to," _
                + "cl_sortkey," _
                + "cl_timestamp)" _
                + " values({0}, '{1}', '{2}', NOW())", _
                PageID, ArticleCategory, ArticleTitle) ', "Now.ToString("yyyy-mm-dd hh:mm:ss"))

            Exec(conn, SQL)
        End If

    End Sub

    Function BlankOutDelimitedText(ByVal inputText As String, ByVal startDelim As String, ByVal endDelim As String) As String
        Dim startDelimStartPos As Integer
        Dim endDelimPos As Integer
        Dim leftText As String
        Dim rightText As String
        Dim middleText As String
        Dim outputText As String

        'inputText = "yed|| [_imagemarker_ 5]" + Chr(10) + Chr(13) + "[_imagemarker_ 6]dsdsd"
        'startDelim = "["
        'endDelim = "]"

        ' Look for an HTML comment.
        outputText = inputText

        startDelimStartPos = InStr(outputText, startDelim)
        While (startDelimStartPos > 0)
            ' Find the end of the comment.
            endDelimPos = InStr(startDelimStartPos + startDelim.Length, outputText, endDelim)
            If (endDelimPos > 0) Then
                ' Hack out the left and right of the comment.
                leftText = Left(outputText, startDelimStartPos - 1)
                rightText = Right(outputText, Len(outputText) - endDelimPos - Len(endDelim) + 1)

                ' Make a string of spaces the same size as the cut string.
                ' This keeps things positionally stable.
                Dim spcCount As Integer
                spcCount = endDelimPos - startDelimStartPos + Len(endDelim)

                middleText = ""
                While (spcCount > 0)
                    middleText = middleText + " "
                    spcCount = spcCount - 1
                End While

                ' Reassemble without the comment.
                outputText = leftText + middleText + rightText
            End If

            ' Look for the next HTML comment.
            startDelimStartPos = InStr(startDelimStartPos + 1, outputText, startDelim)
        End While

        ' Sanity check to make sure input and output are the same length!
        If (Len(inputText) <> Len(outputText)) Then
            'System.Console.WriteLine(inputText)
            'System.Console.WriteLine(outputText)

            Throw New Exception("BlankOutDelimitedText() produced a positionally-unstable result")
        End If

        BlankOutDelimitedText = outputText
    End Function

    Function GetMetadataValue(ByVal inputText As String, ByVal tagName As String) As String
        Dim startPos As Integer = InStr(inputText, "[" + tagName + "=")

        ' Hack out the parameter value.
        If (startPos > 0) Then
            Dim endPos As Integer = InStr(startPos, inputText, "]")

            If (endPos > 0) Then
                GetMetadataValue = Mid(inputText, _
                                       startPos + Len(tagName) + 2, _
                                       endPos - startPos - Len(tagName) - 2)
            Else
                GetMetadataValue = Nothing
            End If
        Else
            GetMetadataValue = Nothing
        End If
    End Function

    Sub DeleteExistingArticle(ByVal conn As MySqlConnection, ByVal PageID As Integer)
        Dim SQL As String
        Dim reader As MySqlDataReader
        Dim work As String

        ' Remove any existing category records for this sucker.
        SQL = String.Format("delete from " + GetTableName(TablePrefix, "categorylinks") _
            + " where cl_from = {0}", PageID)

        Exec(conn, SQL)

        ' Remove the search index entries for this dude.
        SQL = String.Format("delete from " + GetTableName(TablePrefix, "searchindex") _
            + " where si_page = {0}", PageID)

        Exec(conn, SQL)

        ' Remove the page links.
        SQL = String.Format("delete from " + GetTableName(TablePrefix, "pagelinks") _
            + " where pl_from = {0}", PageID)

        Exec(conn, SQL)

        ' Remove text related to the page.
        work = ""
        reader = Query(conn, "select rev_text_id from " + GetTableName(TablePrefix, "revision") + _
                             " where rev_page='" + PageID.ToString() + "'")

        While (reader.Read())
            If work <> "" Then
                work = work + ","
            End If
            work += reader.GetInt32(0).ToString()
        End While
        reader.Close()

        If work <> "" Then
            SQL = String.Format("delete from " + GetTableName(TablePrefix, "text") _
                + " where old_id in (" + work + ")")
        End If

        Exec(conn, SQL)

        ' Remove the revisions.
        SQL = String.Format("delete from " + GetTableName(TablePrefix, "revision") _
            + " where rev_page = {0}", PageID)

        Exec(conn, SQL)

        ' Finally, remove the page record for this dude.
        SQL = String.Format("delete from " + GetTableName(TablePrefix, "page") _
            + " where page_id = {0}", PageID)

        Exec(conn, SQL)
    End Sub

    Function ParseCommandFile(ByVal CommandFilename As String) As ArrayList
        Dim lineContent As String
        Dim reader As StreamReader
        Dim result As ArrayList = New ArrayList

        ' Read the entire damn file!
        reader = New StreamReader(CommandFilename)

        lineContent = reader.ReadLine
        While (Not lineContent Is Nothing)
            Dim equalPos As Integer
            Dim leftOf As String
            Dim rightOf As String

            equalPos = lineContent.IndexOf("=")
            If (equalPos > 0) Then
                leftOf = lineContent.Substring(0, equalPos)
                rightOf = lineContent.Substring(equalPos + 1)
                result.Add("/" + leftOf.Trim)
                result.Add(rightOf.Trim)
            End If

            lineContent = reader.ReadLine
        End While

        ' Cleanup
        reader.Close()
        ParseCommandFile = result
    End Function

    Function ParseBool(ByVal val As String) As Boolean
        If (val = "") Then
            Return False
        Else
            If val = "true" Or val = "TRUE" Then
                Return True
            Else
                Boolean.Parse(val)
            End If
        End If
    End Function
End Module
